In [350]:
import logging
from pathlib import Path
from matplotlib import rc
from IPython.display import display
from pySHOC import shocCampaign, shocHDU
# ensure we return shocCampaign when slicing
shocCampaign.set_returned_type(None) # FIXME! remove the need for this
# setup logging
rootlog = logging.getLogger()
rootlog.setLevel(logging.INFO)
In [351]:
# Load data
root_folder = Path('/media/Oceanus/UCT/Observing/data/sources/J1928-5001')
fig_folder = Path('/home/hannes/Documents/papers/dev/J1928/figures')
rc('savefig', directory=fig_folder)
run = shocCampaign.load_dir(root_folder / 'SHOC/raw')
In [352]:
# Some of the frames are labelled incorrectly.
# OBSTYPE discovery + grouping
g, idx = run.group_by(shocHDU.guess_obstype, return_index=True)
# obstype = run.guess_obstype(plot=True) # to display stats for obstype decision
tables = g.pprint();
# notice `shocHDU.guess_obstype` identifies frames correctly from the
# distribution of pixel values, despite incorrect header info!
⎪ shocObsGroups ⎪ ⎪instrument = SHOC 1 ⎪ ⎪filters.B = ∅ ⎪ ⎪ ⎪f…⎪ ⎪ readout ⎪ timing ⎪ ⎪# ⎪ filename ⎪ tel ⎪ target ⎪obstype ⎪A ⎪ n ⎪ ishape ⎪ bin ⎪γₚᵣₑ ⎪ mode ⎪ t0 ⎪ tExp ⎪ duration ⎪ ⎪0 ⎪SHA_20150715.0002 ⎪74in ⎪SKYFLAT ⎪flat ⎪∅ ⎪ 15⎪(128, 128) ⎪8x8 ⎪ 1.0⎪1 MHz CON ⎪2015-07-15 16:07:32.0 ⎪ 0.14642⎪ 00ʰ00ᵐ02.3ˢ⎪ ⎫ ⎪1 ⎪20130616.0002 ⎪None ⎪None ⎪None ⎪∅ ⎪ 50⎪(128, 128) ⎪8x8 ⎪ 2.4⎪1 MHz CON ⎪2013-06-16 16:02:16.9 ⎪ 0.14642⎪ 00ʰ00ᵐ07.7ˢ⎪ ⎪ ⎪2 ⎪20130617.0001 ⎪None ⎪None ⎪None ⎪∅ ⎪ 100⎪(128, 128) ⎪8x8 ⎪ 2.4⎪1 MHz CON ⎪2013-06-17 16:02:29.9 ⎪ 0.14642⎪ 00ʰ00ᵐ15.3ˢ⎪ ⎪ ⎪3 ⎪202130615.0002 ⎪None ⎪None ⎪None ⎪∅ ⎪ 50⎪(128, 128) ⎪8x8 ⎪ 2.4⎪1 MHz CON ⎪2013-06-15 16:05:53.9 ⎪ 0.14642⎪ 00ʰ00ᵐ07.7ˢ⎪ ⎪ ⎪4 ⎪SHA_20150606.0008 ⎪74in ⎪ ⎪ ⎪∅ ⎪ 10⎪(128, 128) ⎪8x8 ⎪ 1.0⎪3 MHz CON ⎪2015-06-06 16:20:40.0 ⎪ 10.0⎪ 00ʰ01ᵐ40.1ˢ⎪ ⎪ ⎪5 ⎪SHA_20150607.0003 ⎪74in ⎪ ⎪flat ⎪∅ ⎪ 30⎪(128, 128) ⎪8x8 ⎪ 1.0⎪3 MHz CON ⎪2015-06-07 15:57:58.0 ⎪ 0.0558⎪ 00ʰ00ᵐ01.9ˢ⎪ ⎬ flat ⎪6 ⎪SHA_20150904.0012 ⎪74in ⎪SKYFLAT ⎪flat ⎪I ⎪ 15⎪(128, 128) ⎪8x8 ⎪ 1.0⎪1 MHz CON ⎪2015-09-04 16:34:43.0 ⎪ 0.14642⎪ 00ʰ00ᵐ02.3ˢ⎪ ⎪ ⎪7 ⎪SHA_20150904.0022 ⎪74in ⎪SKYFLAT ⎪flat ⎪∅ ⎪ 15⎪(128, 128) ⎪8x8 ⎪ 1.0⎪1 MHz CON ⎪2015-09-04 16:39:23.0 ⎪ 0.14642⎪ 00ʰ00ᵐ02.3ˢ⎪ ⎪ ⎪8 ⎪SHA_20160706.0004 ⎪74in ⎪ ⎪flat ⎪∅ ⎪ 405⎪(85, 85) ⎪12x12 ⎪ 2.4⎪1 MHz CON ⎪2016-07-06 16:14:41.0 ⎪ 0.3⎪ 00ʰ02ᵐ04.2ˢ⎪ ⎪ ⎪9 ⎪SHA_20160707.0004 ⎪74in ⎪ ⎪flat ⎪∅ ⎪ 633⎪(85, 85) ⎪12x12 ⎪ 1.0⎪1 MHz CON ⎪2016-07-07 16:11:08.0 ⎪ 0.11847⎪ 00ʰ01ᵐ19.3ˢ⎪ ⎪ ⎪10 ⎪SHA_20160710.0003 ⎪74in ⎪ ⎪flat ⎪∅ ⎪ 500⎪(128, 128) ⎪8x8 ⎪ 1.0⎪1 MHz CON ⎪2016-07-10 16:06:48.0 ⎪ 0.14642⎪ 00ʰ01ᵐ16.6ˢ⎪ ⎪ ⎪11 ⎪SHA_20160710.0005 ⎪74in ⎪ ⎪flat ⎪∅ ⎪ 500⎪(85, 85) ⎪12x12 ⎪ 1.0⎪1 MHz CON ⎪2016-07-10 16:10:53.0 ⎪ 0.11847⎪ 00ʰ01ᵐ02.6ˢ⎪ ⎭ ⎪ ⎪ ⎪ ⎪ ⎪ ⎪ ⎪ 2323⎪ ⎪ ⎪ ⎪ ⎪ ⎪ ⎪ 00ʰ08ᵐ02.2ˢ⎪ ⎪0 ⎪20130616.0030 ⎪None ⎪None ⎪None ⎪∅ ⎪ 35000⎪(128, 128) ⎪8x8 ⎪ 2.4⎪3 MHz EM: 50 ⎪2013-06-16 21:45:55.9 ⎪ 0.1⎪ 01ʰ02ᵐ16.6ˢ ⎪ ⎫ ⎪1 ⎪20130616.0031 ⎪None ⎪None ⎪None ⎪∅ ⎪ 35000⎪(128, 128) ⎪8x8 ⎪ 2.4⎪3 MHz EM: 50 ⎪2013-06-16 22:48:41.9 ⎪ 0.1⎪ 01ʰ02ᵐ16.6ˢ ⎪ ⎪ ⎪2 ⎪20130617.0030 ⎪None ⎪None ⎪None ⎪∅ ⎪ 35000⎪(128, 128) ⎪8x8 ⎪ 2.4⎪3 MHz EM: 50 ⎪2013-06-17 21:49:58.9 ⎪ 0.1⎪ 01ʰ02ᵐ16.6ˢ ⎪ ⎪ ⎪3 ⎪20130617.0031 ⎪None ⎪None ⎪None ⎪∅ ⎪ 35000⎪(128, 128) ⎪8x8 ⎪ 2.4⎪3 MHz EM: 50 ⎪2013-06-17 22:52:57.9 ⎪ 0.1⎪ 01ʰ02ᵐ16.6ˢ ⎪ ⎪ ⎪4 ⎪20130618.0030 ⎪None ⎪None ⎪None ⎪∅ ⎪ 35000⎪(128, 128) ⎪8x8 ⎪ 2.4⎪3 MHz EM: 125 ⎪2013-06-18 22:41:04.9 ⎪ 0.1⎪ 01ʰ02ᵐ16.6ˢ ⎪ ⎪ ⎪5 ⎪20130618.0031 ⎪None ⎪None ⎪None ⎪∅ ⎪ 22999⎪(128, 128) ⎪8x8 ⎪ 2.4⎪3 MHz EM: 125 ⎪2013-06-18 23:43:58.9 ⎪ 0.1⎪ 00ʰ40ᵐ55.4ˢ ⎪ ⎪ ⎪6 ⎪202130615.0030 ⎪None ⎪None ⎪None ⎪∅ ⎪ 20000⎪(128, 128) ⎪8x8 ⎪ 2.4⎪1 MHz EM: 50 ⎪2013-06-15 23:15:00.9 ⎪ 0.14862⎪ 00ʰ51ᵐ47.6ˢ ⎪ ⎪ ⎪7 ⎪202130615.0031 ⎪None ⎪None ⎪None ⎪∅ ⎪ 20000⎪(128, 128) ⎪8x8 ⎪ 2.4⎪1 MHz EM: 50 ⎪2013-06-16 00:07:07.9 ⎪ 0.14862⎪ 00ʰ51ᵐ47.6ˢ ⎪ ⎪ ⎪8 ⎪202130615.0032 ⎪None ⎪None ⎪None ⎪∅ ⎪ 10000⎪(128, 128) ⎪8x8 ⎪ 2.4⎪1 MHz EM: 50 ⎪2013-06-16 01:01:08.9 ⎪ 0.14862⎪ 00ʰ25ᵐ53.8ˢ ⎪ ⎪ ⎪9 ⎪SHA_20150606.0300 ⎪74in ⎪CTCVJ1928-5001 ⎪object ⎪∅ ⎪ 4824⎪(128, 128) ⎪8x8 ⎪ 1.0⎪3 MHz CON ⎪2015-06-07 01:25:01.0 ⎪ 0.49324⎪ 00ʰ40ᵐ12.0ˢ*⎪ ⎬ object ⎪10 ⎪SHA_20150715.0200 ⎪74in ⎪CXOUJ1723-3734 ⎪object ⎪∅ ⎪ 3584⎪(128, 128) ⎪8x8 ⎪ 1.0⎪1 MHz CON ⎪2015-07-16 01:43:03.0 ⎪ 0.99324⎪ 00ʰ59ᵐ44.0ˢ*⎪ ⎪ ⎪11 ⎪SHA_20150904.0201 ⎪74in ⎪A 48 ⎪object ⎪I ⎪ 7201⎪(128, 128) ⎪8x8 ⎪ 1.0⎪1 MHz CON ⎪2015-09-04 19:35:00.0 ⎪ 0.99324⎪ 02ʰ00ᵐ01.0ˢ*⎪ ⎪ ⎪12 ⎪SHA_20150904.0202 ⎪74in ⎪A 48 ⎪object ⎪∅ ⎪ 9001⎪(128, 128) ⎪8x8 ⎪ 1.0⎪1 MHz CON ⎪2015-09-04 22:40:00.0 ⎪ 0.99324⎪ 02ʰ30ᵐ01.0ˢ*⎪ ⎪ ⎪13 ⎪SHA_20150905.0101 ⎪74in ⎪CTCV J1928-5001 ⎪object ⎪∅ ⎪ 12600⎪(128, 128) ⎪8x8 ⎪ 1.0⎪1 MHz CON ⎪2015-09-05 17:43:00.0 ⎪ 0.99324⎪ 03ʰ30ᵐ00.0ˢ*⎪ ⎪ ⎪14 ⎪SHA_20160706.0007 ⎪74in ⎪ ⎪flat ⎪∅ ⎪ 18253⎪(85, 85) ⎪12x12 ⎪ 4.9⎪1 MHz EM: 50 ⎪2016-07-06 23:23:08.0 ⎪ 0.5⎪ 02ʰ34ᵐ09.9ˢ ⎪ ⎪ ⎪15 ⎪SHA_20160707.0030 ⎪74in ⎪ ⎪object ⎪∅ ⎪ 4301⎪(85, 85) ⎪12x12 ⎪ 4.9⎪1 MHz EM: 50 ⎪2016-07-07 22:18:55.0 ⎪ 0.5⎪ 00ʰ36ᵐ19.6ˢ ⎪ ⎪ ⎪16 ⎪SHA_20160707.0031 ⎪74in ⎪ ⎪object ⎪∅ ⎪ 28000⎪(85, 85) ⎪12x12 ⎪ 4.9⎪1 MHz EM: 50 ⎪2016-07-07 22:55:21.0 ⎪ 0.25⎪ 01ʰ59ᵐ49.3ˢ ⎪ ⎪ ⎪17 ⎪SHA_20160711.0001 ⎪74in ⎪ ⎪object ⎪∅ ⎪ 37505⎪(128, 128) ⎪8x8 ⎪ 4.9⎪1 MHz EM: 65 ⎪2016-07-11 22:51:50.0 ⎪ 0.25⎪ 02ʰ40ᵐ29.8ˢ ⎪ ⎪ ⎪18 ⎪SHA_20171002.0010 ⎪40in ⎪J1928 ⎪object ⎪∅ ⎪ 670⎪(128, 128) ⎪8x8 ⎪ 2.4⎪1 MHz CON ⎪2017-10-02 18:35:38.0 ⎪ 0.5⎪ 00ʰ05ᵐ39.5ˢ ⎪ ⎪ ⎪19 ⎪SHA_20171002.0011 ⎪40in ⎪J1928 ⎪object ⎪∅ ⎪ 12498⎪(128, 128) ⎪8x8 ⎪ 2.4⎪1 MHz CON ⎪2017-10-02 18:59:44.0 ⎪ 0.5⎪ 01ʰ45ᵐ33.5ˢ ⎪ ⎭ ⎪ ⎪ ⎪ ⎪ ⎪ ⎪ ⎪ 386436⎪ ⎪ ⎪ ⎪ ⎪ ⎪ ⎪ 27ʰ23ᵐ46.9ˢ⎪ ⎪0 ⎪20130616.0032 ⎪None ⎪None ⎪None ⎪∅ ⎪ 1⎪(128, 128) ⎪8x8 ⎪ 2.4⎪3 MHz EM: 5 ⎪2013-06-17 00:08:16.9 ⎪ 0.1⎪ 00ʰ00ᵐ00.1ˢ ⎪ ⎫ ⎪1 ⎪20130617.0032 ⎪None ⎪None ⎪None ⎪∅ ⎪ 1⎪(128, 128) ⎪8x8 ⎪ 2.4⎪3 MHz CON ⎪2013-06-18 00:04:42.9 ⎪ 0.1⎪ 00ʰ00ᵐ00.1ˢ ⎪ ⎬ bad ⎪2 ⎪SHA_20150904.0203 ⎪74in ⎪A 48 ⎪object ⎪∅ ⎪ 1⎪(128, 128) ⎪8x8 ⎪ 1.0⎪1 MHz CON ⎪1970-01-01 00:00:00.0 ⎪ 0.99324⎪ 00ʰ00ᵐ01.0ˢ*⎪ ⎪ ⎪3 ⎪SHA_20150904.0204 ⎪74in ⎪A 48 ⎪object ⎪∅ ⎪ 1⎪(128, 128) ⎪8x8 ⎪ 1.0⎪1 MHz CON ⎪1970-01-01 00:00:00.0 ⎪ 0.99324⎪ 00ʰ00ᵐ01.0ˢ*⎪ ⎭ ⎪ ⎪ ⎪ ⎪ ⎪ ⎪ ⎪ 4⎪ ⎪ ⎪ ⎪ ⎪ ⎪ ⎪ 00ʰ00ᵐ02.2ˢ⎪ ⎪0 ⎪20130618.0104 ⎪None ⎪None ⎪None ⎪∅ ⎪ 100⎪(128, 128) ⎪8x8 ⎪ 2.4⎪3 MHz EM: 50 ⎪2013-06-19 02:13:32.0 ⎪ 0.0566⎪ 00ʰ00ᵐ06.3ˢ⎪ ⎫ ⎪1 ⎪20130618.0105 ⎪None ⎪None ⎪None ⎪∅ ⎪ 100⎪(128, 128) ⎪8x8 ⎪ 2.4⎪3 MHz EM: 125 ⎪2013-06-19 02:14:01.0 ⎪ 0.0566⎪ 00ʰ00ᵐ06.3ˢ⎪ ⎪ ⎪2 ⎪202130615.0101 ⎪None ⎪None ⎪None ⎪∅ ⎪ 50⎪(128, 128) ⎪8x8 ⎪ 2.4⎪1 MHz EM: 50 ⎪2013-06-16 04:29:56.9 ⎪ 0.14862⎪ 00ʰ00ᵐ07.8ˢ⎪ ⎪ ⎪3 ⎪SHA_20150607.0010 ⎪74in ⎪ ⎪bias ⎪∅ ⎪ 30⎪(128, 128) ⎪8x8 ⎪ 1.0⎪3 MHz CON ⎪2015-06-07 16:01:12.0 ⎪ 0.0558⎪ 00ʰ00ᵐ01.9ˢ⎪ ⎪ ⎪4 ⎪SHA_20150607.0011 ⎪74in ⎪ ⎪bias ⎪∅ ⎪ 30⎪(128, 128) ⎪8x8 ⎪ 1.0⎪3 MHz CON ⎪2015-06-07 16:01:21.0 ⎪ 0.0558⎪ 00ʰ00ᵐ01.9ˢ⎪ ⎪ ⎪5 ⎪SHA_20150607.0012 ⎪74in ⎪ ⎪bias ⎪∅ ⎪ 30⎪(128, 128) ⎪8x8 ⎪ 1.0⎪3 MHz CON ⎪2015-06-07 16:01:38.0 ⎪ 0.0558⎪ 00ʰ00ᵐ01.9ˢ⎪ ⎪ ⎪6 ⎪SHA_20150715.0051 ⎪74in ⎪BIAS ⎪bias ⎪∅ ⎪ 15⎪(128, 128) ⎪8x8 ⎪ 1.0⎪1 MHz CON ⎪2015-07-16 03:00:02.0 ⎪ 0.14642⎪ 00ʰ00ᵐ02.3ˢ⎪ ⎬ dark ⎪7 ⎪SHA_20150904.0002 ⎪74in ⎪BIAS ⎪bias ⎪I ⎪ 15⎪(128, 128) ⎪8x8 ⎪ 1.0⎪1 MHz CON ⎪2015-09-04 16:01:18.0 ⎪ 0.14642⎪ 00ʰ00ᵐ02.3ˢ⎪ ⎪ ⎪8 ⎪SHA_20160712.0105 ⎪74in ⎪ ⎪dark ⎪∅ ⎪ 1000⎪(128, 128) ⎪8x8 ⎪ 2.4⎪1 MHz CON ⎪2016-07-13 05:33:59.0 ⎪ 0.14642⎪ 00ʰ02ᵐ33.2ˢ⎪ ⎪ ⎪9 ⎪SHA_20160712.0109 ⎪74in ⎪ ⎪dark ⎪∅ ⎪ 1000⎪(128, 128) ⎪8x8 ⎪ 4.9⎪1 MHz EM: 65 ⎪2016-07-13 05:46:45.0 ⎪ 0.14862⎪ 00ʰ02ᵐ35.4ˢ⎪ ⎪ ⎪10 ⎪SHA_20160712.0114 ⎪74in ⎪ ⎪dark ⎪∅ ⎪ 1000⎪(102, 102) ⎪10x10 ⎪ 4.9⎪1 MHz EM: 50 ⎪2016-07-13 06:00:42.0 ⎪ 0.12067⎪ 00ʰ02ᵐ07.4ˢ⎪ ⎪ ⎪11 ⎪SHA_20160712.0115 ⎪74in ⎪ ⎪dark ⎪∅ ⎪ 1000⎪(85, 85) ⎪12x12 ⎪ 1.0⎪1 MHz CON ⎪2016-07-13 06:04:13.0 ⎪ 0.1002⎪ 00ʰ01ᵐ47.0ˢ⎪ ⎪ ⎪12 ⎪SHA_20160712.0116 ⎪74in ⎪ ⎪dark ⎪∅ ⎪ 1000⎪(85, 85) ⎪12x12 ⎪ 2.4⎪1 MHz CON ⎪2016-07-13 06:06:08.0 ⎪ 0.1002⎪ 00ʰ01ᵐ47.0ˢ⎪ ⎪ ⎪13 ⎪SHA_20160712.0117 ⎪74in ⎪ ⎪dark ⎪∅ ⎪ 1000⎪(85, 85) ⎪12x12 ⎪ 4.9⎪1 MHz EM: 50 ⎪2016-07-13 06:08:52.0 ⎪ 0.10242⎪ 00ʰ01ᵐ49.2ˢ⎪ ⎭ ⎪ ⎪ ⎪ ⎪ ⎪ ⎪ ⎪ 6370⎪ ⎪ ⎪ ⎪ ⎪ ⎪ ⎪ 00ʰ13ᵐ09.8ˢ⎪ * tExp: timing.trigger.is_gps_loop * t0: timing.trigger.is_gps
In [353]:
# Apply the guessed observation type labels (flat / dark / bad)
# Note this updates the attributes of the shocHDU's, but leaves the fits
# headers untouched
for obstype, r in g.items():
# set attributes on HDU objects - not yet in header
r.set_attrs(obstype=obstype)
# remove bad files
g.pop('bad', None)
# set the target name
run_src = g['object']
run_src.set_attrs(target='CTCV J1928-5001')
# add telescope info for old data. We will need this later
for obstype in ['object', 'flat']:
run = g[obstype]
is74in = np.equal(run.attrs('telescope'), None)
run[is74in].set_attrs(telescope='74in')
# print target observations
run_src.sort_by('date').pprint(); # todo: list by date; export to latex
⎪ shocCampaign ⎪ ⎪camera = SHOC 1 ⎪ ⎪target = CTCV J1928-5001 ⎪ ⎪obstype = object ⎪ ⎪B = ∅ ⎪ ⎪ ⎪f…⎪ ⎪ readout ⎪ timing ⎪ ⎪# ⎪ filename ⎪ tel ⎪A ⎪ n ⎪ ishape ⎪ bin ⎪γₚᵣₑ ⎪ mode ⎪ t0 ⎪ tExp ⎪ duration ⎪ ⎪0 ⎪202130615.0030 ⎪74in ⎪∅ ⎪ 20000⎪(128, 128) ⎪8x8 ⎪ 2.4⎪1 MHz EM: 50 ⎪2013-06-15 23:15:00.9 ⎪ 0.14862 ⎪ 00ʰ51ᵐ47.6ˢ⎪ ⎪1 ⎪20130616.0030 ⎪74in ⎪∅ ⎪ 35000⎪(128, 128) ⎪8x8 ⎪ 2.4⎪3 MHz EM: 50 ⎪2013-06-16 21:45:55.9 ⎪ 0.1 ⎪ 01ʰ02ᵐ16.6ˢ⎪ ⎪2 ⎪20130616.0031 ⎪74in ⎪∅ ⎪ 35000⎪(128, 128) ⎪8x8 ⎪ 2.4⎪3 MHz EM: 50 ⎪2013-06-16 22:48:41.9 ⎪ 0.1 ⎪ 01ʰ02ᵐ16.6ˢ⎪ ⎪3 ⎪202130615.0031 ⎪74in ⎪∅ ⎪ 20000⎪(128, 128) ⎪8x8 ⎪ 2.4⎪1 MHz EM: 50 ⎪2013-06-16 00:07:07.9 ⎪ 0.14862 ⎪ 00ʰ51ᵐ47.6ˢ⎪ ⎪4 ⎪202130615.0032 ⎪74in ⎪∅ ⎪ 10000⎪(128, 128) ⎪8x8 ⎪ 2.4⎪1 MHz EM: 50 ⎪2013-06-16 01:01:08.9 ⎪ 0.14862 ⎪ 00ʰ25ᵐ53.8ˢ⎪ ⎪5 ⎪20130617.0030 ⎪74in ⎪∅ ⎪ 35000⎪(128, 128) ⎪8x8 ⎪ 2.4⎪3 MHz EM: 50 ⎪2013-06-17 21:49:58.9 ⎪ 0.1 ⎪ 01ʰ02ᵐ16.6ˢ⎪ ⎪6 ⎪20130617.0031 ⎪74in ⎪∅ ⎪ 35000⎪(128, 128) ⎪8x8 ⎪ 2.4⎪3 MHz EM: 50 ⎪2013-06-17 22:52:57.9 ⎪ 0.1 ⎪ 01ʰ02ᵐ16.6ˢ⎪ ⎪7 ⎪20130618.0030 ⎪74in ⎪∅ ⎪ 35000⎪(128, 128) ⎪8x8 ⎪ 2.4⎪3 MHz EM: 125 ⎪2013-06-18 22:41:04.9 ⎪ 0.1 ⎪ 01ʰ02ᵐ16.6ˢ⎪ ⎪8 ⎪20130618.0031 ⎪74in ⎪∅ ⎪ 22999⎪(128, 128) ⎪8x8 ⎪ 2.4⎪3 MHz EM: 125 ⎪2013-06-18 23:43:58.9 ⎪ 0.1 ⎪ 00ʰ40ᵐ55.4ˢ⎪ ⎪9 ⎪SHA_20150606.0300 ⎪74in ⎪∅ ⎪ 4824⎪(128, 128) ⎪8x8 ⎪ 1.0⎪3 MHz CON ⎪2015-06-07 01:25:01.0* ⎪ 0.49324*⎪ 00ʰ40ᵐ12.0ˢ⎪ ⎪10 ⎪SHA_20150715.0200 ⎪74in ⎪∅ ⎪ 3584⎪(128, 128) ⎪8x8 ⎪ 1.0⎪1 MHz CON ⎪2015-07-16 01:43:03.0* ⎪ 0.99324*⎪ 00ʰ59ᵐ44.0ˢ⎪ ⎪11 ⎪SHA_20150904.0201 ⎪74in ⎪I ⎪ 7201⎪(128, 128) ⎪8x8 ⎪ 1.0⎪1 MHz CON ⎪2015-09-04 19:35:00.0* ⎪ 0.99324*⎪ 02ʰ00ᵐ01.0ˢ⎪ ⎪12 ⎪SHA_20150904.0202 ⎪74in ⎪∅ ⎪ 9001⎪(128, 128) ⎪8x8 ⎪ 1.0⎪1 MHz CON ⎪2015-09-04 22:40:00.0* ⎪ 0.99324*⎪ 02ʰ30ᵐ01.0ˢ⎪ ⎪13 ⎪SHA_20150905.0101 ⎪74in ⎪∅ ⎪ 12600⎪(128, 128) ⎪8x8 ⎪ 1.0⎪1 MHz CON ⎪2015-09-05 17:43:00.0* ⎪ 0.99324*⎪ 03ʰ30ᵐ00.0ˢ⎪ ⎪14 ⎪SHA_20160706.0007 ⎪74in ⎪∅ ⎪ 18253⎪(85, 85) ⎪12x12 ⎪ 4.9⎪1 MHz EM: 50 ⎪2016-07-06 23:23:08.0 ⎪ 0.5 ⎪ 02ʰ34ᵐ09.9ˢ⎪ ⎪15 ⎪SHA_20160707.0030 ⎪74in ⎪∅ ⎪ 4301⎪(85, 85) ⎪12x12 ⎪ 4.9⎪1 MHz EM: 50 ⎪2016-07-07 22:18:55.0 ⎪ 0.5 ⎪ 00ʰ36ᵐ19.6ˢ⎪ ⎪16 ⎪SHA_20160707.0031 ⎪74in ⎪∅ ⎪ 28000⎪(85, 85) ⎪12x12 ⎪ 4.9⎪1 MHz EM: 50 ⎪2016-07-07 22:55:21.0 ⎪ 0.25 ⎪ 01ʰ59ᵐ49.3ˢ⎪ ⎪17 ⎪SHA_20160711.0001 ⎪74in ⎪∅ ⎪ 37505⎪(128, 128) ⎪8x8 ⎪ 4.9⎪1 MHz EM: 65 ⎪2016-07-11 22:51:50.0 ⎪ 0.25 ⎪ 02ʰ40ᵐ29.8ˢ⎪ ⎪18 ⎪SHA_20171002.0010 ⎪40in ⎪∅ ⎪ 670⎪(128, 128) ⎪8x8 ⎪ 2.4⎪1 MHz CON ⎪2017-10-02 18:35:38.0 ⎪ 0.5 ⎪ 00ʰ05ᵐ39.5ˢ⎪ ⎪19 ⎪SHA_20171002.0011 ⎪40in ⎪∅ ⎪ 12498⎪(128, 128) ⎪8x8 ⎪ 2.4⎪1 MHz CON ⎪2017-10-02 18:59:44.0 ⎪ 0.5 ⎪ 01ʰ45ᵐ33.5ˢ⎪ ⎪ ⎪ ⎪ ⎪ ⎪ 386436⎪ ⎪ ⎪ ⎪ ⎪ ⎪ ⎪ 27ʰ23ᵐ46.9ˢ⎪ * t0: timing.trigger.is_gps * tExp: timing.trigger.is_gps_loop
In [179]:
# Science images thumbnail grid
fig, axes, _ = run_src.sort_by('date').thumbnails()
fig.set_size_inches(9, 7)
fig
Out[179]:
In [354]:
# Match calibration frames
ATT_EQUAL_DARK = ('instrument', 'binning', # subrect
'readout.preAmpGain', 'readout.outAmp.mode', 'readout.frq') # <-- 'readout'
ATT_CLOSE_DARK = ('readout.outAmp.emGain', 'timing.t_expose')
ATT_DARK = ATT_EQUAL_DARK, ATT_CLOSE_DARK
# DEBIAS
# need to debias flats & science observations
obs = g['flat'].join(g['object'])
gobs, gdark = obs.match(g['dark'], *ATT_DARK, keep_nulls=False, report=True)
# compute master darks
mbias = gdark.merge_combine(np.median)
⎪Matched Observations ⎪ ⎪exact: ('instrument', 'binning', 'readout.preAmpGain', 'readout.outAmp.mode', 'readout.frq') ⎪ ⎪closest: ('readout.outAmp.emGain', 'timing.t_expose') ⎪ ⎪B = ∅ ⎪ ⎪ ⎪f…⎪ ⎪ readout ⎪ timing ⎪ ⎪ filename ⎪ tel ⎪ target ⎪obstype ⎪A ⎪ n ⎪ ishape ⎪ mode ⎪ t0 ⎪ duration ⎪ ⎪ ⎪ ⎪group 0: 12x12; 1.0; CON; 1; 0; 0.1002 ⎪ ⎪SHA_20160712.0115 ⎪74in ⎪ ⎪dark ⎪∅ ⎪ 1000⎪(85, 85) ⎪1 MHz CON ⎪2016-07-13 06:04:13.0 ⎪ 00ʰ01ᵐ47.0ˢ⎪ ⎪SHA_20160707.0004 ⎪74in ⎪ ⎪flat ⎪∅ ⎪ 633⎪(85, 85) ⎪1 MHz CON ⎪2016-07-07 16:11:08.0 ⎪ 00ʰ01ᵐ19.3ˢ⎪ ⎪SHA_20160710.0005 ⎪74in ⎪ ⎪flat ⎪∅ ⎪ 500⎪(85, 85) ⎪1 MHz CON ⎪2016-07-10 16:10:53.0 ⎪ 00ʰ01ᵐ02.6ˢ⎪ ⎪ ⎪ ⎪group 1: 8x8; 2.4; EM; 1; 50; 0.14862 ⎪ ⎪202130615.0101 ⎪None ⎪None ⎪dark ⎪∅ ⎪ 50⎪(128, 128) ⎪1 MHz EM: 50 ⎪2013-06-16 04:29:56.9 ⎪ 00ʰ00ᵐ07.8ˢ⎪ ⎪202130615.0030 ⎪74in ⎪CTCV J1928-5001 ⎪object ⎪∅ ⎪ 20000⎪(128, 128) ⎪1 MHz EM: 50 ⎪2013-06-15 23:15:00.9 ⎪ 00ʰ51ᵐ47.6ˢ⎪ ⎪202130615.0031 ⎪74in ⎪CTCV J1928-5001 ⎪object ⎪∅ ⎪ 20000⎪(128, 128) ⎪1 MHz EM: 50 ⎪2013-06-16 00:07:07.9 ⎪ 00ʰ51ᵐ47.6ˢ⎪ ⎪202130615.0032 ⎪74in ⎪CTCV J1928-5001 ⎪object ⎪∅ ⎪ 10000⎪(128, 128) ⎪1 MHz EM: 50 ⎪2013-06-16 01:01:08.9 ⎪ 00ʰ25ᵐ53.8ˢ⎪ ⎪ ⎪ ⎪group 2: 8x8; 1.0; CON; 1; 0; 0.14642 ⎪ ⎪SHA_20150715.0051 ⎪74in ⎪BIAS ⎪dark ⎪∅ ⎪ 15⎪(128, 128) ⎪1 MHz CON ⎪2015-07-16 03:00:02.0 ⎪ 00ʰ00ᵐ02.3ˢ⎪ ⎪SHA_20150904.0002 ⎪74in ⎪BIAS ⎪dark ⎪I ⎪ 15⎪(128, 128) ⎪1 MHz CON ⎪2015-09-04 16:01:18.0 ⎪ 00ʰ00ᵐ02.3ˢ⎪ ⎪SHA_20150715.0002 ⎪74in ⎪SKYFLAT ⎪flat ⎪∅ ⎪ 15⎪(128, 128) ⎪1 MHz CON ⎪2015-07-15 16:07:32.0 ⎪ 00ʰ00ᵐ02.3ˢ⎪ ⎪SHA_20150904.0012 ⎪74in ⎪SKYFLAT ⎪flat ⎪I ⎪ 15⎪(128, 128) ⎪1 MHz CON ⎪2015-09-04 16:34:43.0 ⎪ 00ʰ00ᵐ02.3ˢ⎪ ⎪SHA_20150904.0022 ⎪74in ⎪SKYFLAT ⎪flat ⎪∅ ⎪ 15⎪(128, 128) ⎪1 MHz CON ⎪2015-09-04 16:39:23.0 ⎪ 00ʰ00ᵐ02.3ˢ⎪ ⎪SHA_20160710.0003 ⎪74in ⎪ ⎪flat ⎪∅ ⎪ 500⎪(128, 128) ⎪1 MHz CON ⎪2016-07-10 16:06:48.0 ⎪ 00ʰ01ᵐ16.6ˢ⎪ ⎪SHA_20150715.0200 ⎪74in ⎪CTCV J1928-5001 ⎪object ⎪∅ ⎪ 3584⎪(128, 128) ⎪1 MHz CON ⎪2015-07-16 01:43:03.0 ⎪ 00ʰ59ᵐ44.0ˢ⎪ ⎪SHA_20150904.0201 ⎪74in ⎪CTCV J1928-5001 ⎪object ⎪I ⎪ 7201⎪(128, 128) ⎪1 MHz CON ⎪2015-09-04 19:35:00.0 ⎪ 02ʰ00ᵐ01.0ˢ⎪ ⎪SHA_20150904.0202 ⎪74in ⎪CTCV J1928-5001 ⎪object ⎪∅ ⎪ 9001⎪(128, 128) ⎪1 MHz CON ⎪2015-09-04 22:40:00.0 ⎪ 02ʰ30ᵐ01.0ˢ⎪ ⎪SHA_20150905.0101 ⎪74in ⎪CTCV J1928-5001 ⎪object ⎪∅ ⎪ 12600⎪(128, 128) ⎪1 MHz CON ⎪2015-09-05 17:43:00.0 ⎪ 03ʰ30ᵐ00.0ˢ⎪ ⎪ ⎪ ⎪group 3: 8x8; 4.9; EM; 1; 65; 0.14862 ⎪ ⎪SHA_20160712.0109 ⎪74in ⎪ ⎪dark ⎪∅ ⎪ 1000⎪(128, 128) ⎪1 MHz EM: 65 ⎪2016-07-13 05:46:45.0 ⎪ 00ʰ02ᵐ35.4ˢ⎪ ⎪SHA_20160711.0001 ⎪74in ⎪CTCV J1928-5001 ⎪object ⎪∅ ⎪ 37505⎪(128, 128) ⎪1 MHz EM: 65 ⎪2016-07-11 22:51:50.0 ⎪ 02ʰ40ᵐ29.8ˢ⎪ ⎪ ⎪ ⎪group 4: 12x12; 4.9; EM; 1; 50; 0.10242 ⎪ ⎪SHA_20160712.0117 ⎪74in ⎪ ⎪dark ⎪∅ ⎪ 1000⎪(85, 85) ⎪1 MHz EM: 50 ⎪2016-07-13 06:08:52.0 ⎪ 00ʰ01ᵐ49.2ˢ⎪ ⎪SHA_20160706.0007 ⎪74in ⎪CTCV J1928-5001 ⎪object ⎪∅ ⎪ 18253⎪(85, 85) ⎪1 MHz EM: 50 ⎪2016-07-06 23:23:08.0 ⎪ 02ʰ34ᵐ09.9ˢ⎪ ⎪SHA_20160707.0030 ⎪74in ⎪CTCV J1928-5001 ⎪object ⎪∅ ⎪ 4301⎪(85, 85) ⎪1 MHz EM: 50 ⎪2016-07-07 22:18:55.0 ⎪ 00ʰ36ᵐ19.6ˢ⎪ ⎪SHA_20160707.0031 ⎪74in ⎪CTCV J1928-5001 ⎪object ⎪∅ ⎪ 28000⎪(85, 85) ⎪1 MHz EM: 50 ⎪2016-07-07 22:55:21.0 ⎪ 01ʰ59ᵐ49.3ˢ⎪ ⎪ ⎪ ⎪group 5: 8x8; 2.4; EM; 3; 50; 0.0566 ⎪ ⎪20130618.0104 ⎪None ⎪None ⎪dark ⎪∅ ⎪ 100⎪(128, 128) ⎪3 MHz EM: 50 ⎪2013-06-19 02:13:32.0 ⎪ 00ʰ00ᵐ06.3ˢ⎪ ⎪20130616.0030 ⎪74in ⎪CTCV J1928-5001 ⎪object ⎪∅ ⎪ 35000⎪(128, 128) ⎪3 MHz EM: 50 ⎪2013-06-16 21:45:55.9 ⎪ 01ʰ02ᵐ16.6ˢ⎪ ⎪20130616.0031 ⎪74in ⎪CTCV J1928-5001 ⎪object ⎪∅ ⎪ 35000⎪(128, 128) ⎪3 MHz EM: 50 ⎪2013-06-16 22:48:41.9 ⎪ 01ʰ02ᵐ16.6ˢ⎪ ⎪20130617.0030 ⎪74in ⎪CTCV J1928-5001 ⎪object ⎪∅ ⎪ 35000⎪(128, 128) ⎪3 MHz EM: 50 ⎪2013-06-17 21:49:58.9 ⎪ 01ʰ02ᵐ16.6ˢ⎪ ⎪20130617.0031 ⎪74in ⎪CTCV J1928-5001 ⎪object ⎪∅ ⎪ 35000⎪(128, 128) ⎪3 MHz EM: 50 ⎪2013-06-17 22:52:57.9 ⎪ 01ʰ02ᵐ16.6ˢ⎪ ⎪ ⎪ ⎪group 6: 8x8; 2.4; EM; 3; 125; 0.0566 ⎪ ⎪20130618.0105 ⎪None ⎪None ⎪dark ⎪∅ ⎪ 100⎪(128, 128) ⎪3 MHz EM: 125 ⎪2013-06-19 02:14:01.0 ⎪ 00ʰ00ᵐ06.3ˢ⎪ ⎪20130618.0030 ⎪74in ⎪CTCV J1928-5001 ⎪object ⎪∅ ⎪ 35000⎪(128, 128) ⎪3 MHz EM: 125 ⎪2013-06-18 22:41:04.9 ⎪ 01ʰ02ᵐ16.6ˢ⎪ ⎪20130618.0031 ⎪74in ⎪CTCV J1928-5001 ⎪object ⎪∅ ⎪ 22999⎪(128, 128) ⎪3 MHz EM: 125 ⎪2013-06-18 23:43:58.9 ⎪ 00ʰ40ᵐ55.4ˢ⎪ ⎪ ⎪ ⎪group 7: 12x12; 2.4; CON; 1; 0; 0.1002 ⎪ ⎪SHA_20160712.0116 ⎪74in ⎪ ⎪dark ⎪∅ ⎪ 1000⎪(85, 85) ⎪1 MHz CON ⎪2016-07-13 06:06:08.0 ⎪ 00ʰ01ᵐ47.0ˢ⎪ ⎪SHA_20160706.0004 ⎪74in ⎪ ⎪flat ⎪∅ ⎪ 405⎪(85, 85) ⎪1 MHz CON ⎪2016-07-06 16:14:41.0 ⎪ 00ʰ02ᵐ04.2ˢ⎪ ⎪ ⎪ ⎪group 8: 8x8; 1.0; CON; 3; 0; 0.0558 ⎪ ⎪SHA_20150607.0010 ⎪74in ⎪ ⎪dark ⎪∅ ⎪ 30⎪(128, 128) ⎪3 MHz CON ⎪2015-06-07 16:01:12.0 ⎪ 00ʰ00ᵐ01.9ˢ⎪ ⎪SHA_20150607.0011 ⎪74in ⎪ ⎪dark ⎪∅ ⎪ 30⎪(128, 128) ⎪3 MHz CON ⎪2015-06-07 16:01:21.0 ⎪ 00ʰ00ᵐ01.9ˢ⎪ ⎪SHA_20150607.0012 ⎪74in ⎪ ⎪dark ⎪∅ ⎪ 30⎪(128, 128) ⎪3 MHz CON ⎪2015-06-07 16:01:38.0 ⎪ 00ʰ00ᵐ01.9ˢ⎪ ⎪SHA_20150606.0008 ⎪74in ⎪ ⎪flat ⎪∅ ⎪ 10⎪(128, 128) ⎪3 MHz CON ⎪2015-06-06 16:20:40.0 ⎪ 00ʰ01ᵐ40.1ˢ⎪ ⎪SHA_20150607.0003 ⎪74in ⎪ ⎪flat ⎪∅ ⎪ 30⎪(128, 128) ⎪3 MHz CON ⎪2015-06-07 15:57:58.0 ⎪ 00ʰ00ᵐ01.9ˢ⎪ ⎪SHA_20150606.0300 ⎪74in ⎪CTCV J1928-5001 ⎪object ⎪∅ ⎪ 4824⎪(128, 128) ⎪3 MHz CON ⎪2015-06-07 01:25:01.0 ⎪ 00ʰ40ᵐ12.0ˢ⎪ ⎪ ⎪ ⎪group 9: 8x8; 2.4; CON; 1; 0; 0.14642 ⎪ ⎪SHA_20160712.0105 ⎪74in ⎪ ⎪dark ⎪∅ ⎪ 1000⎪(128, 128) ⎪1 MHz CON ⎪2016-07-13 05:33:59.0 ⎪ 00ʰ02ᵐ33.2ˢ⎪ ⎪20130616.0002 ⎪74in ⎪None ⎪flat ⎪∅ ⎪ 50⎪(128, 128) ⎪1 MHz CON ⎪2013-06-16 16:02:16.9 ⎪ 00ʰ00ᵐ07.7ˢ⎪ ⎪20130617.0001 ⎪74in ⎪None ⎪flat ⎪∅ ⎪ 100⎪(128, 128) ⎪1 MHz CON ⎪2013-06-17 16:02:29.9 ⎪ 00ʰ00ᵐ15.3ˢ⎪ ⎪202130615.0002 ⎪74in ⎪None ⎪flat ⎪∅ ⎪ 50⎪(128, 128) ⎪1 MHz CON ⎪2013-06-15 16:05:53.9 ⎪ 00ʰ00ᵐ07.7ˢ⎪ ⎪SHA_20171002.0010 ⎪40in ⎪CTCV J1928-5001 ⎪object ⎪∅ ⎪ 670⎪(128, 128) ⎪1 MHz CON ⎪2017-10-02 18:35:38.0 ⎪ 00ʰ05ᵐ39.5ˢ⎪ ⎪SHA_20171002.0011 ⎪40in ⎪CTCV J1928-5001 ⎪object ⎪∅ ⎪ 12498⎪(128, 128) ⎪1 MHz CON ⎪2017-10-02 18:59:44.0 ⎪ 01ʰ45ᵐ33.5ˢ⎪ * t0: timing.trigger.is_gps * tExp: timing.trigger.is_gps_loop
In [ ]:
# Notice that i have previously placed all the correct dark and flat
# observation files in the one folder. This is actually not necessary. If
# you already have a folder where you store all your calibration data, you can
# read all those files and select the correct ones with a few lines of code
# similar to what was done above.
# For example:
# darks = shocCampaign.load_dir('/media/Oceanus/UCT/Observing/data/darks')
#_ = run_src.match(darks, *ATT_DARK, keep_nulls=False, report=True)
In [10]:
# display
fig, _ = mbias.to_list().sort_by('date').thumbnails(title=('date', 'readout'))
fig
Out[10]:
In [355]:
from obstools.stats import median_scaled_median
# Science files are large and won't all fit into RAM, so needs careful handling.
# use `set_calibrators` to do calibration arithmetic on the fly when accessing
# data via `calibrated` attribute
gobj = gobs.select_by(obstype='object')
gobj.set_calibrators(mbias)
# Flat fields are small enough volume that we can safely read them into RAM.
gflat = gobs.select_by(obstype='flat')
gflat = gflat.subtract(mbias)
# Match calibrated flat fields to science observations. This grouping will be
# different to that of `gflat` above since we are now matching for closest
# dates
ATT_EQUAL_FLAT = ('telescope', 'instrument', 'binning', 'filters')
ATT_CLOSE_FLAT = ('date', )
ATT_FLAT = ATT_EQUAL_FLAT, ATT_CLOSE_FLAT
gobj, gflat = g['object'].match(gflat.to_list(), *ATT_FLAT, keep_nulls=False,
report=True)
# flat field: median scale each image, then median combine images
mflat = gflat.merge_combine(median_scaled_median)
#
gobj.set_calibrators(flats=mflat)
⎪Matched Observations ⎪ ⎪exact: ('telescope', 'instrument', 'binning', 'filters') ⎪ ⎪closest: ('date',) ⎪ ⎪None ⎪ ⎪ ⎪ readout ⎪ timing ⎪ ⎪ filename ⎪ target ⎪obstype ⎪ n ⎪ ishape ⎪γₚᵣₑ ⎪ mode ⎪ t0 ⎪ tExp ⎪ duration ⎪ ⎪ ⎪ ⎪group 0: 74in; 8x8; Filters('∅', '∅'); 2015-07-15 ⎪ ⎪SHA_20150715.0002 ⎪SKYFLAT ⎪flat ⎪ 15⎪(128, 128) ⎪ 1.0⎪1 MHz CON ⎪2015-07-15 16:07:32.0 ⎪ 0.14642⎪ 00ʰ00ᵐ02.3ˢ⎪ ⎪SHA_20150715.0200 ⎪CTCV J1928-5001 ⎪object ⎪ 3584⎪(128, 128) ⎪ 1.0⎪1 MHz CON ⎪2015-07-16 01:43:03.0 ⎪ 0.99324⎪ 00ʰ59ᵐ44.0ˢ⎪ ⎪ ⎪ ⎪group 1: 74in; 8x8; Filters('∅', '∅'); 2015-09-04 ⎪ ⎪SHA_20150904.0022 ⎪SKYFLAT ⎪flat ⎪ 15⎪(128, 128) ⎪ 1.0⎪1 MHz CON ⎪2015-09-04 16:39:23.0 ⎪ 0.14642⎪ 00ʰ00ᵐ02.3ˢ⎪ ⎪SHA_20150904.0202 ⎪CTCV J1928-5001 ⎪object ⎪ 9001⎪(128, 128) ⎪ 1.0⎪1 MHz CON ⎪2015-09-04 22:40:00.0 ⎪ 0.99324⎪ 02ʰ30ᵐ01.0ˢ⎪ ⎪SHA_20150905.0101 ⎪CTCV J1928-5001 ⎪object ⎪ 12600⎪(128, 128) ⎪ 1.0⎪1 MHz CON ⎪2015-09-05 17:43:00.0 ⎪ 0.99324⎪ 03ʰ30ᵐ00.0ˢ⎪ ⎪ ⎪ ⎪group 2: 74in; 8x8; Filters('∅', '∅'); 2016-07-10 ⎪ ⎪SHA_20160710.0003 ⎪ ⎪flat ⎪ 500⎪(128, 128) ⎪ 1.0⎪1 MHz CON ⎪2016-07-10 16:06:48.0 ⎪ 0.14642⎪ 00ʰ01ᵐ16.6ˢ⎪ ⎪SHA_20160711.0001 ⎪CTCV J1928-5001 ⎪object ⎪ 37505⎪(128, 128) ⎪ 4.9⎪1 MHz EM: 65 ⎪2016-07-11 22:51:50.0 ⎪ 0.25⎪ 02ʰ40ᵐ29.8ˢ⎪ ⎪ ⎪ ⎪group 3: 74in; 8x8; Filters('∅', '∅'); 2015-06-07 ⎪ ⎪SHA_20150607.0003 ⎪ ⎪flat ⎪ 30⎪(128, 128) ⎪ 1.0⎪3 MHz CON ⎪2015-06-07 15:57:58.0 ⎪ 0.0558⎪ 00ʰ00ᵐ01.9ˢ⎪ ⎪SHA_20150606.0300 ⎪CTCV J1928-5001 ⎪object ⎪ 4824⎪(128, 128) ⎪ 1.0⎪3 MHz CON ⎪2015-06-07 01:25:01.0 ⎪ 0.49324⎪ 00ʰ40ᵐ12.0ˢ⎪ ⎪ ⎪ ⎪group 4: 74in; 8x8; Filters('∅', '∅'); 2013-06-16 ⎪ ⎪20130616.0002 ⎪None ⎪flat ⎪ 50⎪(128, 128) ⎪ 2.4⎪1 MHz CON ⎪2013-06-16 16:02:16.9 ⎪ 0.14642⎪ 00ʰ00ᵐ07.7ˢ⎪ ⎪20130616.0030 ⎪CTCV J1928-5001 ⎪object ⎪ 35000⎪(128, 128) ⎪ 2.4⎪3 MHz EM: 50 ⎪2013-06-16 21:45:55.9 ⎪ 0.1⎪ 01ʰ02ᵐ16.6ˢ⎪ ⎪20130616.0031 ⎪CTCV J1928-5001 ⎪object ⎪ 35000⎪(128, 128) ⎪ 2.4⎪3 MHz EM: 50 ⎪2013-06-16 22:48:41.9 ⎪ 0.1⎪ 01ʰ02ᵐ16.6ˢ⎪ ⎪202130615.0031 ⎪CTCV J1928-5001 ⎪object ⎪ 20000⎪(128, 128) ⎪ 2.4⎪1 MHz EM: 50 ⎪2013-06-16 00:07:07.9 ⎪ 0.14862⎪ 00ʰ51ᵐ47.6ˢ⎪ ⎪202130615.0032 ⎪CTCV J1928-5001 ⎪object ⎪ 10000⎪(128, 128) ⎪ 2.4⎪1 MHz EM: 50 ⎪2013-06-16 01:01:08.9 ⎪ 0.14862⎪ 00ʰ25ᵐ53.8ˢ⎪ ⎪ ⎪ ⎪group 5: 74in; 8x8; Filters('∅', '∅'); 2013-06-17 ⎪ ⎪20130617.0001 ⎪None ⎪flat ⎪ 100⎪(128, 128) ⎪ 2.4⎪1 MHz CON ⎪2013-06-17 16:02:29.9 ⎪ 0.14642⎪ 00ʰ00ᵐ15.3ˢ⎪ ⎪20130617.0030 ⎪CTCV J1928-5001 ⎪object ⎪ 35000⎪(128, 128) ⎪ 2.4⎪3 MHz EM: 50 ⎪2013-06-17 21:49:58.9 ⎪ 0.1⎪ 01ʰ02ᵐ16.6ˢ⎪ ⎪20130617.0031 ⎪CTCV J1928-5001 ⎪object ⎪ 35000⎪(128, 128) ⎪ 2.4⎪3 MHz EM: 50 ⎪2013-06-17 22:52:57.9 ⎪ 0.1⎪ 01ʰ02ᵐ16.6ˢ⎪ ⎪20130618.0030 ⎪CTCV J1928-5001 ⎪object ⎪ 35000⎪(128, 128) ⎪ 2.4⎪3 MHz EM: 125 ⎪2013-06-18 22:41:04.9 ⎪ 0.1⎪ 01ʰ02ᵐ16.6ˢ⎪ ⎪20130618.0031 ⎪CTCV J1928-5001 ⎪object ⎪ 22999⎪(128, 128) ⎪ 2.4⎪3 MHz EM: 125 ⎪2013-06-18 23:43:58.9 ⎪ 0.1⎪ 00ʰ40ᵐ55.4ˢ⎪ ⎪ ⎪ ⎪group 6: 74in; 8x8; Filters('∅', '∅'); 2013-06-15 ⎪ ⎪202130615.0002 ⎪None ⎪flat ⎪ 50⎪(128, 128) ⎪ 2.4⎪1 MHz CON ⎪2013-06-15 16:05:53.9 ⎪ 0.14642⎪ 00ʰ00ᵐ07.7ˢ⎪ ⎪202130615.0030 ⎪CTCV J1928-5001 ⎪object ⎪ 20000⎪(128, 128) ⎪ 2.4⎪1 MHz EM: 50 ⎪2013-06-15 23:15:00.9 ⎪ 0.14862⎪ 00ʰ51ᵐ47.6ˢ⎪ ⎪ ⎪ ⎪group 7: 40in; 8x8; Filters('∅', '∅') ⎪ ⎪NO MATCH ⎪ ⎪ ⎪ ⎪ ⎪ ⎪ ⎪ ⎪ ⎪ ⎪ ⎪SHA_20171002.0010 ⎪CTCV J1928-5001 ⎪object ⎪ 670⎪(128, 128) ⎪ 2.4⎪1 MHz CON ⎪2017-10-02 18:35:38.0 ⎪ 0.5⎪ 00ʰ05ᵐ39.5ˢ⎪ ⎪SHA_20171002.0011 ⎪CTCV J1928-5001 ⎪object ⎪ 12498⎪(128, 128) ⎪ 2.4⎪1 MHz CON ⎪2017-10-02 18:59:44.0 ⎪ 0.5⎪ 01ʰ45ᵐ33.5ˢ⎪ ⎪ ⎪ ⎪group 8: 74in; 12x12; Filters('∅', '∅'); 2016-07-07 ⎪ ⎪SHA_20160707.0004 ⎪ ⎪flat ⎪ 633⎪(85, 85) ⎪ 1.0⎪1 MHz CON ⎪2016-07-07 16:11:08.0 ⎪ 0.11847⎪ 00ʰ01ᵐ19.3ˢ⎪ ⎪SHA_20160707.0030 ⎪CTCV J1928-5001 ⎪object ⎪ 4301⎪(85, 85) ⎪ 4.9⎪1 MHz EM: 50 ⎪2016-07-07 22:18:55.0 ⎪ 0.5⎪ 00ʰ36ᵐ19.6ˢ⎪ ⎪SHA_20160707.0031 ⎪CTCV J1928-5001 ⎪object ⎪ 28000⎪(85, 85) ⎪ 4.9⎪1 MHz EM: 50 ⎪2016-07-07 22:55:21.0 ⎪ 0.25⎪ 01ʰ59ᵐ49.3ˢ⎪ ⎪ ⎪ ⎪group 9: 74in; 12x12; Filters('∅', '∅'); 2016-07-06 ⎪ ⎪SHA_20160706.0004 ⎪ ⎪flat ⎪ 405⎪(85, 85) ⎪ 2.4⎪1 MHz CON ⎪2016-07-06 16:14:41.0 ⎪ 0.3⎪ 00ʰ02ᵐ04.2ˢ⎪ ⎪SHA_20160706.0007 ⎪CTCV J1928-5001 ⎪object ⎪ 18253⎪(85, 85) ⎪ 4.9⎪1 MHz EM: 50 ⎪2016-07-06 23:23:08.0 ⎪ 0.5⎪ 02ʰ34ᵐ09.9ˢ⎪ ⎪ ⎪ ⎪group 10: 74in; 8x8; Filters('I', '∅'); 2015-09-04 ⎪ ⎪SHA_20150904.0012 ⎪SKYFLAT ⎪flat ⎪ 15⎪(128, 128) ⎪ 1.0⎪1 MHz CON ⎪2015-09-04 16:34:43.0 ⎪ 0.14642⎪ 00ʰ00ᵐ02.3ˢ⎪ ⎪SHA_20150904.0201 ⎪CTCV J1928-5001 ⎪object ⎪ 7201⎪(128, 128) ⎪ 1.0⎪1 MHz CON ⎪2015-09-04 19:35:00.0 ⎪ 0.99324⎪ 02ʰ00ᵐ01.0ˢ⎪ * t0: timing.trigger.is_gps * tExp: timing.trigger.is_gps_loop
In [26]:
# display master flats
fig, _ = mflat.to_list().sort_by('date').thumbnails(title='date')
fig
Out[26]:
In [7]:
# show image grid (calibrated images)
# gobj.set_calibrators(mbias) # mflat
orun = gobj.to_list().sort_by('date')
fig, _ = orun.thumbnails(calibrated=True)
fig
Out[7]:
In [356]:
# The flats for 20150606.0300.fits seem to introduce artifacts rather than
# remove them. This is most likely due to small number statistics since
# there are only 30 frames in the flat field observation..
orun = gobj.to_list().sort_by('date')
orun[9].calibrated.flat = None
# Also don't have calibration images for 1.0m data, which can't be helped
In [358]:
# Image registration & mosaic
import cmasher as cmr
# align images
reg = orun.coalign(plot=False)
# plot
mp = reg.mosaic(alpha=0.35, cmap='cmr.dusk', number_sources=True)
mp.fig
INFO:core.shocOldHDU:Computing median of 67 images (exposure depth of 10.0 seconds) for sample image from '202130615.0030.fits' INFO:core.shocOldHDU:Computing median of 99 images (exposure depth of 10.0 seconds) for sample image from '20130616.0030.fits' INFO:core.shocOldHDU:Computing median of 99 images (exposure depth of 10.0 seconds) for sample image from '20130616.0031.fits' INFO:core.shocOldHDU:Computing median of 67 images (exposure depth of 10.0 seconds) for sample image from '202130615.0031.fits' INFO:core.shocOldHDU:Computing median of 67 images (exposure depth of 10.0 seconds) for sample image from '202130615.0032.fits' INFO:core.shocOldHDU:Computing median of 99 images (exposure depth of 10.0 seconds) for sample image from '20130617.0030.fits' INFO:core.shocOldHDU:Computing median of 99 images (exposure depth of 10.0 seconds) for sample image from '20130617.0031.fits' INFO:core.shocOldHDU:Computing median of 99 images (exposure depth of 10.0 seconds) for sample image from '20130618.0030.fits' INFO:core.shocOldHDU:Computing median of 99 images (exposure depth of 10.0 seconds) for sample image from '20130618.0031.fits' INFO:core.shocNewHDU:Computing median of 20 images (exposure depth of 10.0 seconds) for sample image from 'SHA_20150606.0300.fits' INFO:core.shocNewHDU:Computing median of 10 images (exposure depth of 10.0 seconds) for sample image from 'SHA_20150715.0200.fits' INFO:core.shocNewHDU:Computing median of 10 images (exposure depth of 10.0 seconds) for sample image from 'SHA_20150904.0202.fits' INFO:core.shocNewHDU:Computing median of 10 images (exposure depth of 10.0 seconds) for sample image from 'SHA_20150904.0201.fits' INFO:core.shocNewHDU:Computing median of 10 images (exposure depth of 10.0 seconds) for sample image from 'SHA_20150905.0101.fits' INFO:core.shocFlatHDU:Computing median of 20 images (exposure depth of 10.0 seconds) for sample image from 'SHA_20160706.0007.fits' INFO:core.shocNewHDU:Computing median of 20 images (exposure depth of 10.0 seconds) for sample image from 'SHA_20160707.0030.fits' INFO:core.shocNewHDU:Computing median of 40 images (exposure depth of 10.0 seconds) for sample image from 'SHA_20160707.0031.fits' INFO:core.shocNewHDU:Computing median of 40 images (exposure depth of 10.0 seconds) for sample image from 'SHA_20160711.0001.fits' INFO:registration.ImageRegister:Aligning 18 images on image 0 INFO:image.registration:Identifying stars INFO:image.registration:Clustering 189 position measurements using: MeanShift(bandwidth=5.94703255887994, bin_seeding=False, cluster_all=False, min_bin_freq=1, n_jobs=None, seeds=None) INFO:image.registration:Identified 23 stars using 189/189 points (0 noise) INFO:image.registration:Identifying stars INFO:image.registration:Clustering 189 position measurements using: MeanShift(bandwidth=5.94703255887994, bin_seeding=False, cluster_all=False, min_bin_freq=1, n_jobs=None, seeds=None) INFO:image.registration:Identified 23 stars using 189/189 points (0 noise) INFO:registration.ImageRegister:Fitting successful 17 / 17 INFO:registration.ImageRegister:Likelihood ratio: 0.99873 INFO:registration.ImageRegister:Keeping same parameters. INFO:image.registration:Measuring cluster centres, frame xy-offsets INFO:image.registration:Ignoring 8 / 23 stars with low (<=25%) detection frequency for frame offset measurement. INFO:image.registration: ⎪ Measured star locations ⎪ ⎪# ⎪ n (%)⎪ x⎪ y⎪ ⎪1 ⎪ 18 (100%)⎪ 55.74 ± 0.09⎪ 92.01 ± 0.18⎪ ⎪2 ⎪ 18 (100%)⎪ 47.37 ± 0.16⎪ 41.55 ± 0.13⎪ ⎪3 ⎪ 18 (100%)⎪ 72.54 ± 0.13⎪ 38.54 ± 0.21⎪ ⎪4 ⎪ 18 (100%)⎪ 57.74 ± 0.15⎪ 28.44 ± 0.19⎪ ⎪5 ⎪ 14 (78%)⎪ 15.83 ± 0.07⎪ 96.26 ± 0.11⎪ ⎪6 ⎪ 13 (72%)⎪ 93.01 ± 0.11⎪ 43.19 ± 0.13⎪ ⎪7 ⎪ 12 (67%)⎪ 27.42 ± 0.52⎪ 49.33 ± 0.91⎪ ⎪8 ⎪ 11 (61%)⎪ 108.03 ± 0.13⎪ 74.48 ± 0.21⎪ ⎪9 ⎪ 8 (44%)⎪ 2.80 ± 0.24⎪ 86.53 ± 0.20⎪ ⎪10 ⎪ 8 (44%)⎪ -4.50 ± 0.12⎪ 29.69 ± 0.28⎪ ⎪11 ⎪ 7 (39%)⎪ 64.59 ± 0.59⎪ 124.95 ± 0.30⎪ ⎪12 ⎪ 7 (39%)⎪ -5.45 ± 0.20⎪ 122.24 ± 0.21⎪ ⎪13 ⎪ 7 (39%)⎪ -12.13 ± 0.14⎪ 41.26 ± 0.24⎪ ⎪14 ⎪ 5 (28%)⎪ 5.44 ± 0.43⎪ 128.03 ± 0.29⎪ ⎪15 ⎪ 5 (28%)⎪ 100.83 ± 0.18⎪ 23.85 ± 0.39⎪ ⎪16 ⎪ 4 (22%)⎪ -28.74 ± 51.63⎪ 74.79 ± 51.63⎪ ⎪17 ⎪ 3 (17%)⎪ 77.75 ± 16.80⎪ 111.20 ± 16.80⎪ ⎪18 ⎪ 3 (17%)⎪ 129.30 ± 33.93⎪ 61.32 ± 33.93⎪ ⎪19 ⎪ 3 (17%)⎪ 132.96 ± 67.83⎪ -2.68 ± 67.83⎪ ⎪20 ⎪ 2 (11%)⎪ -7.93 ± 5.80⎪ 3.64 ± 5.80⎪ ⎪21 ⎪ 2 (11%)⎪ 115.71 ± 61.59⎪ -7.47 ± 61.59⎪ ⎪22 ⎪ 2 (11%)⎪ 143.47 ± 80.92⎪ -18.37 ± 80.92⎪ ⎪23 ⎪ 1 (6%)⎪ 120.39 ± 71.78⎪ -23.16 ± 71.78⎪ ⎪ ⎪ 189 (1050%)⎪ ⎪ ⎪ n_noise = 225/414 (54.3%) INFO:registration.ImageRegister:Likelihood ratio: nan INFO:registration.ImageRegister:Keeping same parameters. INFO:core.shocNewHDU:Computing median of 20 images (exposure depth of 10.0 seconds) for sample image from 'SHA_20171002.0010.fits' INFO:core.shocNewHDU:Computing median of 20 images (exposure depth of 10.0 seconds) for sample image from 'SHA_20171002.0011.fits' INFO:registration.ImageRegister:Aligning 2 images on image 0 INFO:image.registration:Identifying stars INFO:image.registration:Clustering 21 position measurements using: MeanShift(bandwidth=2.343487033951146, bin_seeding=False, cluster_all=False, min_bin_freq=1, n_jobs=None, seeds=None) INFO:image.registration:Identified 13 stars using 21/21 points (0 noise) INFO:image.registration:Identifying stars INFO:image.registration:Clustering 21 position measurements using: MeanShift(bandwidth=2.343487033951146, bin_seeding=False, cluster_all=False, min_bin_freq=1, n_jobs=None, seeds=None) INFO:image.registration:Identified 13 stars using 21/21 points (0 noise) INFO:registration.ImageRegister:Fitting successful 1 / 1 INFO:registration.ImageRegister:Likelihood ratio: 0.99916 INFO:registration.ImageRegister:Keeping same parameters. INFO:image.registration:Measuring cluster centres, frame xy-offsets INFO:image.registration: ⎪ Measured star locations ⎪ ⎪# ⎪ n (%)⎪ x⎪ y⎪ ⎪1 ⎪ 2 (100%)⎪ 57.56 ± 0.02⎪ 116.71 ± 0.01⎪ ⎪2 ⎪ 2 (100%)⎪ 53.74 ± 0.01⎪ 93.79 ± 0.03⎪ ⎪3 ⎪ 2 (100%)⎪ 19.11 ± 0.07⎪ 67.73 ± 0.05⎪ ⎪4 ⎪ 2 (100%)⎪ 97.06 ± 0.04⎪ 66.52 ± 0.07⎪ ⎪5 ⎪ 2 (100%)⎪ 46.95 ± 0.06⎪ 41.71 ± 0.07⎪ ⎪6 ⎪ 2 (100%)⎪ 80.76 ± 0.09⎪ 38.67 ± 0.03⎪ ⎪7 ⎪ 2 (100%)⎪ 81.93 ± 0.09⎪ 29.36 ± 0.03⎪ ⎪8 ⎪ 2 (100%)⎪ 6.56 ± 0.04⎪ 26.95 ± 0.12⎪ ⎪9 ⎪ 1 (50%)⎪ -4.13 ± 0.00⎪ 99.71 ± 0.00⎪ ⎪10 ⎪ 1 (50%)⎪ 30.60 ± 0.00⎪ 88.42 ± 0.00⎪ ⎪11 ⎪ 1 (50%)⎪ 59.49 ± 0.00⎪ 62.37 ± 0.00⎪ ⎪12 ⎪ 1 (50%)⎪ 54.90 ± 0.00⎪ 43.18 ± 0.00⎪ ⎪13 ⎪ 1 (50%)⎪ 1.85 ± 0.00⎪ 26.52 ± 0.00⎪ ⎪ ⎪ 21 (1050%)⎪ ⎪ ⎪ n_noise = 5/26 (19.2%) INFO:registration.ImageRegister:Likelihood ratio: 0.99942 INFO:registration.ImageRegister:Keeping same parameters. WARNING:registration.CoherentPointDrift:CoherentPointDrift optimization with objective 'Model.loss_mle' failed to converge: Desired error not necessarily achieved due to precision loss. INFO:image.registration:Identifying stars INFO:image.registration:Clustering 210 position measurements using: MeanShift(bandwidth=5.177471354078108, bin_seeding=False, cluster_all=False, min_bin_freq=1, n_jobs=None, seeds=None) INFO:image.registration:Identified 32 stars using 210/210 points (0 noise) INFO:registration.ImageRegister:Fitting successful 19 / 19 INFO:registration.ImageRegister:Likelihood ratio: 0.99881 INFO:registration.ImageRegister:Keeping same parameters. INFO:image.registration:Measuring cluster centres, frame xy-offsets INFO:image.registration:Ignoring 19 / 32 stars with low (<=25%) detection frequency for frame offset measurement. INFO:image.registration: ⎪ Measured star locations ⎪ ⎪# ⎪ n (%)⎪ x⎪ y⎪ ⎪1 ⎪ 20 (100%)⎪ 55.73 ± 0.11⎪ 92.04 ± 0.17⎪ ⎪2 ⎪ 20 (100%)⎪ 47.35 ± 0.15⎪ 41.53 ± 0.11⎪ ⎪3 ⎪ 18 (90%)⎪ 72.54 ± 0.13⎪ 38.53 ± 0.22⎪ ⎪4 ⎪ 18 (90%)⎪ 57.74 ± 0.14⎪ 28.44 ± 0.19⎪ ⎪5 ⎪ 14 (70%)⎪ 15.85 ± 0.08⎪ 96.25 ± 0.11⎪ ⎪6 ⎪ 13 (65%)⎪ 93.01 ± 0.10⎪ 43.19 ± 0.15⎪ ⎪7 ⎪ 12 (60%)⎪ 27.42 ± 0.50⎪ 49.32 ± 0.90⎪ ⎪8 ⎪ 11 (55%)⎪ 108.03 ± 0.12⎪ 74.47 ± 0.19⎪ ⎪9 ⎪ 9 (45%)⎪ -4.49 ± 0.19⎪ 29.68 ± 0.26⎪ ⎪10 ⎪ 8 (40%)⎪ 2.80 ± 0.23⎪ 86.54 ± 0.20⎪ ⎪11 ⎪ 7 (35%)⎪ 64.59 ± 0.62⎪ 124.95 ± 0.31⎪ ⎪12 ⎪ 7 (35%)⎪ -5.44 ± 0.23⎪ 122.23 ± 0.22⎪ ⎪13 ⎪ 7 (35%)⎪ -12.13 ± 0.12⎪ 41.26 ± 0.25⎪ ⎪14 ⎪ 5 (25%)⎪ 5.45 ± 61.37⎪ 128.03 ± 61.37⎪ ⎪15 ⎪ 5 (25%)⎪ 100.83 ± 38.56⎪ 23.85 ± 38.56⎪ ⎪16 ⎪ 4 (20%)⎪ -28.74 ± 51.64⎪ 74.79 ± 51.64⎪ ⎪17 ⎪ 4 (20%)⎪ 143.39 ± 80.88⎪ -18.42 ± 80.88⎪ ⎪18 ⎪ 3 (15%)⎪ 77.79 ± 16.83⎪ 111.21 ± 16.83⎪ ⎪19 ⎪ 3 (15%)⎪ 129.30 ± 33.93⎪ 61.31 ± 33.93⎪ ⎪20 ⎪ 3 (15%)⎪ 132.96 ± 67.83⎪ -2.68 ± 67.83⎪ ⎪21 ⎪ 2 (10%)⎪ -7.93 ± 5.80⎪ 3.65 ± 5.80⎪ ⎪22 ⎪ 2 (10%)⎪ 115.70 ± 61.59⎪ -7.48 ± 61.59⎪ ⎪23 ⎪ 2 (10%)⎪ -29.29 ± 6.61⎪ -16.07 ± 6.61⎪ ⎪24 ⎪ 2 (10%)⎪ 32.22 ± 52.88⎪ -73.55 ± 52.88⎪ ⎪25 ⎪ 2 (10%)⎪ 106.91 ± 93.59⎪ -80.27 ± 93.59⎪ ⎪26 ⎪ 2 (10%)⎪ 109.50 ± 105.18⎪ -100.85 ± 105.18⎪ ⎪27 ⎪ 2 (10%)⎪ -57.01 ± 24.58⎪ -106.17 ± 24.58⎪ ⎪28 ⎪ 1 (5%)⎪ -80.72 ± 67.64⎪ 54.57 ± 67.64⎪ ⎪29 ⎪ 1 (5%)⎪ 120.37 ± 71.78⎪ -23.19 ± 71.78⎪ ⎪30 ⎪ 1 (5%)⎪ 60.00 ± 43.94⎪ -27.89 ± 43.94⎪ ⎪31 ⎪ 1 (5%)⎪ 49.87 ± 60.09⎪ -70.30 ± 60.09⎪ ⎪32 ⎪ 1 (5%)⎪ -67.52 ± 19.80⎪ -107.13 ± 19.80⎪ ⎪ ⎪ 210 (1050%)⎪ ⎪ ⎪ n_noise = 430/640 (67.2%) INFO:registration.ImageRegister:Likelihood ratio: 0.99974 INFO:registration.ImageRegister:Keeping same parameters.
Out[358]:
In [359]:
from graphing.imagine import ImageDisplay
def pixel_transform(i: int):
# scale images by source counts for source 0 and median subtract
image = reg[i]
a = image.data / image.counts[sidx[i] == 0]
return a - np.ma.median(a)
# get source indices per frame
sidx = reg.source_indices
# get mean image across stack
g, bs = reg.binned_statistic(image_func=pixel_transform, interpolate=True)
im = ImageDisplay(bs.statistic.T)
im.imagePlot.set_clim(0, 0.012)
im.figure.set_size_inches(8, 8)
im.figure
Out[359]:
